{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost Parameter Tuning for Otto Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第四步：调整树的参数：subsample 和 colsample_bytree\n",
    "(粗调，参数的步长为0.1；下一步是在粗调最佳参数周围，将步长降为0.05，进行精细调整)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先 import 必要的模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# path to where the data lies\n",
    "dpath = './'\n",
    "train = pd.read_csv(dpath +\"FE_train.csv\")\n",
    "#train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variable Identification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Target 分布，看看各类样本分布是否均衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFZVJREFUeJzt3X+wJWV95/H3h5+igoAOFJmBzCCzKpHS4AQhyfoLRTAb+VGSQKmwht3JuqjERFc0rrOKRNhERGo1LrUQQCNI8NdIkFlEMRtLgeGXCEgxCwoTiKAgIi7owHf/OM+FMzN37u2502cux/t+VZ263U8/3ed7rJFPdT/dT6eqkCSpD1vMdgGSpF8fhookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN1vNdgGb23Oe85xauHDhbJchSWPjmmuu+XFVzevSd86FysKFC1m5cuVslyFJYyPJD7v29fKXJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN3PuifpN9ZJ3nzfbJegp6Jq/Pma2S5CeEjxTkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9WakoZLknUluSvK9JOcneVqSRUmuTHJbks8l2ab13batr2rbFw4d572t/dYkrx1qP7i1rUpy4ih/iyRpeiMLlSTzgXcAS6rqhcCWwFHAqcDHqmox8ABwXNvlOOCBqtoL+FjrR5K9236/BRwMfDLJlkm2BD4BHALsDRzd+kqSZsmoL39tBWyXZCvg6cA9wKuAi9r2c4HD2vKhbZ22/cAkae0XVNWjVXUHsArYr31WVdXtVfVL4ILWV5I0S0YWKlX1L8DfAHcyCJMHgWuAn1bVmtZtNTC/Lc8H7mr7rmn9nz3cvs4+G2qXJM2SUV7+2onBmcMi4DeAZzC4VLWumthlA9s2tn2yWpYmWZlk5X333Tdd6ZKkGRrl5a9XA3dU1X1V9SvgC8DvAju2y2EAC4C72/JqYHeAtv1ZwP3D7evss6H29VTVmVW1pKqWzJs3r4/fJkmaxChD5U5g/yRPb2MjBwI3A98A3tD6HAt8uS0vb+u07V+vqmrtR7W7wxYBi4GrgKuBxe1usm0YDOYvH+HvkSRNY2Qv6aqqK5NcBFwLrAGuA84E/hG4IMmHW9tZbZezgE8nWcXgDOWodpybklzIIJDWAMdX1WMASd4GrGBwZ9nZVXXTqH6PJGl6I33zY1UtA5at03w7gzu31u37CHDkBo5zMnDyJO2XAJdseqWSpD74RL0kqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN9OGSpLnJtm2Lb8iyTuS7Dj60iRJ46bLmcrngceS7AWcBSwCPjvSqiRJY6lLqDxeVWuAw4HTq+qdwG6jLUuSNI66hMqvkhwNHAtc3Nq2Hl1JkqRx1SVU3gIcAJxcVXckWQR8ZrRlSZLG0VZTbUyyJfC+qnrTRFtV3QGcMurCJEnjZ8ozlap6DJiXZJvNVI8kaYxNeabS/AD4VpLlwMMTjVV12qiKkiSNpy6hcnf7bAFsP9pyJEnjbNpQqaoPAiR5RlU9PF1/SdLc1eWJ+gOS3Azc0tZflOSTI69MkjR2utxSfDrwWuAnAFV1A/CyURYlSRpPnSaUrKq71ml6bAS1SJLGXJdQuSvJ7wKVZJsk76JdCptOkh2TXJTk+0luaZfSdk5yWZLb2t+dWt8kOSPJqiTfTbLv0HGObf1vS3LsUPtLktzY9jkjSTby90uSetQlVP4TcDwwH1gNvLitd/Fx4NKqej7wIgZhdCJweVUtBi5v6wCHAIvbZynwtwBJdgaWAS8F9gOWTQRR67N0aL+DO9YlSRqBLnd//Rh448YeOMkODMZe/n07zi+BXyY5FHhF63YucAXwHuBQ4LyqKuA77Sxnt9b3sqq6vx33MuDgJFcAO1TVt1v7ecBhwFc3tlZJUj+mDZU219fbgYXD/avq9dPsuidwH/B3SV4EXAOcAOxaVfe0Y9yTZJfWfz4wPHazurVN1b56knZJ0izp8vDjlxi8R+UrwOMbeex9gbdX1ZVJPs6Tl7omM9l4SM2gff0DJ0sZXCZjjz32mKpmSdIm6BIqj1TVGTM49mpgdVVd2dYvYhAqP0qyWztL2Q24d6j/7kP7L2DwJP9qnrxcNtF+RWtfMEn/9VTVmcCZAEuWLJk0eCRJm67LQP3Hkyxrd27tO/GZbqeq+lcGd449rzUdCNwMLGfwbhba3y+35eXAMe0usP2BB9tlshXAQUl2agP0BwEr2raHkuzf7vo6ZuhYkqRZ0OVMZR/gzcCrePLyV7X16bwd+Ps2y/HtDN7NsgVwYZLjgDuBI1vfS4DXAauAX7S+VNX9SU4Crm79PjQxaA+8FTgH2I7BAL2D9JI0i7qEyuHAnu3urY1SVdcDSybZdOAkfYsN3KpcVWcDZ0/SvhJ44cbWJUkajS6Xv24Adhx1IZKk8dflTGVX4PtJrgYenWjscEuxJGmO6RIqy0ZehSTp10KXJ+q/mWRX4Hda01VVde9U+0iS5qYu71P5I+AqBndp/RFwZZI3jLowSdL46XL56y+B35k4O0kyD/gag4cZJUl6Qpe7v7ZY53LXTzruJ0maY7qcqVyaZAVwflv/Y3zIUJI0iS4D9e9OcgTw+wwmcTyzqr448sokSWOny9T3p1bVe4AvTNImSdITuoyNvGaStkP6LkSSNP42eKaS5K3Afwb2TPLdoU3bA98adWGSpPEz1eWvzzIYkP8Ia79c66GhWYIlSXrCBi9/VdWDVfUD4P3Av1bVD4FFwJuSOMGkJGk9XcZUPg88lmQvBq8VXsTgLEaSpLV0CZXHq2oNcARwelW9E9httGVJksZRl1D5VZKjGbyu9+LWtvXoSpIkjasuofIW4ADg5Kq6I8ki4DOjLUuSNI66PFF/M/COofU7gFNGWZQkaTx1eaL+DqDWba+qPUdSkSRpbHWZUHLJ0PLTGLxXZefRlCNJGmfTjqlU1U+GPv9SVacDr9oMtUmSxkyXy1/7Dq1uweDMZfuRVSRJGltdLn99dGh5DXAHg9cKS5K0li53f71ycxQiSRp/046pJPmr4bm+kuyU5MOjLUuSNI66PPx4SFX9dGKlqh4AXje6kiRJ46pLqGyZZNuJlSTbAdtO0V+SNEd1Gaj/DHB5kr9j8BDknwDnjrQqSdJY6jJQ/9/bmx9fDQQ4qapWjLwySdLY6XKmQlVdClw64lokSWOuy5iKJEmdGCqSpN5sMFSSXN7+nrr5ypEkjbOpxlR2S/Jy4PVJLmAwSP+Eqrp2pJVJksbOVKHyAeBEYAFw2jrbCmcqliStY4OhUlUXARcl+a9VddJmrEmSNKa6PKdyUpLXAy9rTVdU1cWjLUuSNI66TCj5EeAE4Ob2OaG1SZK0li4PP/4B8OKqehwgybnAdcB7R1mYJGn8dH1OZceh5WdtzBck2TLJdUkubuuLklyZ5LYkn0uyTWvftq2vatsXDh3jva391iSvHWo/uLWtSnLixtQlSepfl1D5CHBdknPaWco1wF9txHecANwytH4q8LGqWgw8ABzX2o8DHqiqvYCPtX4k2Rs4Cvgt4GDgky2otgQ+ARwC7A0c3fpKkmbJtKFSVecD+wNfaJ8DquqCLgdPsoDB5bP/1dbD4Fbki1qXc4HD2vKhPDn78UXAga3/ocAFVfVoVd0BrAL2a59VVXV7Vf0SuKD1lSTNkq4TSt4DLJ/B8U8H/guwfVt/NvDTqlrT1lcD89vyfOCu9n1rkjzY+s8HvjN0zOF97lqn/aWTFZFkKbAUYI899pjBz5AkdTGyub+S/Dvg3qq6Zrh5kq41zbaNbV+/serMqlpSVUvmzZs3RdWSpE3R6Uxlhn6PwRQvrwOeBuzA4MxlxyRbtbOVBcDdrf9qYHdgdZKtGNwQcP9Q+4ThfTbULkmaBVOeqSTZIsn3ZnLgqnpvVS2oqoUMBtq/XlVvBL4BvKF1Oxb4clte3tZp279eVdXaj2p3hy0CFgNXAVcDi9vdZNu075jJJTpJUk+mPFOpqseT3JBkj6q6s6fvfA9wQZIPM3je5azWfhbw6SSrGJyhHNVquCnJhQwevFwDHF9VjwEkeRuwAtgSOLuqbuqpRknSDHS5/LUbcFOSq4CHJxqr6vVdv6SqrgCuaMu3M7hza90+jwBHbmD/k4GTJ2m/BLikax2SpNHqEiofHHkVkqRfC10mlPxmkt8EFlfV15I8ncHlJkmS1tJlQsn/yOBhxP/ZmuYDXxplUZKk8dTlOZXjGdwe/DOAqroN2GWURUmSxlOXUHm0TYMCQHuGZNKHDCVJc1uXUPlmkvcB2yV5DfAPwFdGW5YkaRx1CZUTgfuAG4E/ZXAL7/tHWZQkaTx1ufvr8Tbl/ZUMLnvd2p50lyRpLdOGSpI/AD4F/F8GkzguSvKnVfXVURcnSRovXR5+/CjwyqpaBZDkucA/AoaKJGktXcZU7p0IlOZ24N4R1SNJGmMbPFNJckRbvCnJJcCFDMZUjmQwQ7AkSWuZ6vLXHw4t/wh4eVu+D9hpZBVJksbWBkOlqt6yOQuRJI2/Lnd/LQLeDiwc7r8xU99LkuaGLnd/fYnBC7S+Ajw+2nIkSeOsS6g8UlVnjLwSSdLY6xIqH0+yDPjfwKMTjVV17ciqkiSNpS6hsg/wZuBVPHn5q9q6JElP6BIqhwN7Dk9/L0nSZLo8UX8DsOOoC5Ekjb8uZyq7At9PcjVrj6l4S7EkaS1dQmXZyKuQJP1a6PI+lW9ujkIkSeOvyxP1D/HkO+m3AbYGHq6qHUZZmCRp/HQ5U9l+eD3JYcB+I6tIkjS2utz9tZaq+hI+oyJJmkSXy19HDK1uASzhycthkiQ9ocvdX8PvVVkD/AA4dCTVSJLGWpcxFd+rIknqZKrXCX9giv2qqk4aQT2SpDE21ZnKw5O0PQM4Dng2YKhIktYy1euEPzqxnGR74ATgLcAFwEc3tJ8kae6ackwlyc7AnwNvBM4F9q2qBzZHYZKk8TPVmMpfA0cAZwL7VNXPN1tVkqSxNNXDj38B/AbwfuDuJD9rn4eS/GzzlCdJGidTjals9NP2kqS5zeCQJPXGUJEk9WZkoZJk9yTfSHJLkpuSnNDad05yWZLb2t+dWnuSnJFkVZLvJtl36FjHtv63JTl2qP0lSW5s+5yRJKP6PZKk6Y3yTGUN8BdV9QJgf+D4JHsDJwKXV9Vi4PK2DnAIsLh9lgJ/C0/c1rwMeCmDKfeXTQRR67N0aL+DR/h7JEnTGFmoVNU9VXVtW34IuAWYz2AyynNbt3OBw9ryocB5NfAdYMckuwGvBS6rqvvbMzKXAQe3bTtU1berqoDzho4lSZoFm2VMJclC4LeBK4Fdq+oeGAQPsEvrNh+4a2i31a1tqvbVk7RP9v1Lk6xMsvK+++7b1J8jSdqAkYdKkmcCnwf+rKqmer5lsvGQmkH7+o1VZ1bVkqpaMm/evOlKliTN0EhDJcnWDALl76vqC635R+3SFe3vva19NbD70O4LgLunaV8wSbskaZaM8u6vAGcBt1TVaUOblgMTd3AdC3x5qP2YdhfY/sCD7fLYCuCgJDu1AfqDgBVt20NJ9m/fdczQsSRJs6DLmx9n6veANwM3Jrm+tb0POAW4MMlxwJ3AkW3bJcDrgFXALxjMiExV3Z/kJODq1u9DVXV/W34rcA6wHfDV9pEkzZKRhUpV/TOTj3sAHDhJ/wKO38CxzgbOnqR9JfDCTShTktQjn6iXJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1ZuxDJcnBSW5NsirJibNdjyTNZVvNdgGbIsmWwCeA1wCrgauTLK+qm2e3Mml23PmhfWa7BD0F7fGBGzfbd437mcp+wKqqur2qfglcABw6yzVJ0pw17qEyH7hraH11a5MkzYKxvvwFZJK2Wq9TshRY2lZ/nuTWkVY1dzwH+PFsF/FUkL85drZL0Pr89zlh2WT/qdwov9m147iHympg96H1BcDd63aqqjOBMzdXUXNFkpVVtWS265Am47/P2THul7+uBhYnWZRkG+AoYPks1yRJc9ZYn6lU1ZokbwNWAFsCZ1fVTbNcliTNWWMdKgBVdQlwyWzXMUd5SVFPZf77nAWpWm9cW5KkGRn3MRVJ0lOIoaIZcXocPVUlOTvJvUm+N9u1zEWGijba0PQ4hwB7A0cn2Xt2q5KecA5w8GwXMVcZKpoJp8fRU1ZV/RNw/2zXMVcZKpoJp8eRNClDRTPRaXocSXOPoaKZ6DQ9jqS5x1DRTDg9jqRJGSraaFW1BpiYHucW4EKnx9FTRZLzgW8Dz0uyOslxs13TXOIT9ZKk3nimIknqjaEiSeqNoSJJ6o2hIknqjaEiSeqNoSJ1lOSxJNcnuSnJDUn+PMkWbduSJGdMse8rkly8+apd7/v/W5J3zdb3a+4Y+zc/SpvR/6uqFwMk2QX4LPAsYFlVrQRWjuqLk2zVng+SntI8U5FmoKruBZYCb8vAE2ciSV7ezmiuT3Jdku3bbjsk+WKSm5N8augs5+cTx03yhiTntOVzkpyW5BvAqRs6bpJ3J7k6yXeTfHDoWH/Z3nnzNeB5m+N/F8kzFWmGqur2Fgy7rLPpXcDxVfWtJM8EHmnt+zF4/8wPgUuBI4CLpvmafwO8uqoeS/KVdY+b5CBgcTt2gOVJXgY8zGD6nN9m8P/za4FrNu0XS9PzTEXaNJPN2Pwt4LQk7wB2HLpsdVV7B81jwPnA73c4/j+0/hs67kHtcx2D4Hg+g5D5t8AXq+oXVfUznJtNm4mhIs1Qkj2Bx4B7h9ur6hTgPwDbAd9J8vyJTescoiZpf9o6fR6e5rgBPlJVL26fvarqrA18nzRyhoo0A0nmAZ8C/ketM4FekudW1Y1VdSqDwfuJUNmvzey8BfDHwD+39h8leUFrP3yK75zsuCuAP2mXw0gyv91E8E/A4Um2a2Mvf9jXb5em4piK1N12Sa4HtgbWAJ8GTpuk358leSWDs5ibga8CBzCYOfcUYB8G/9H/Yut/InAxg7dpfg945ga+f73jVtWjSV4AfDsJwM+BN1XVtUk+B1zPYAzn/2zKD5e6cpZiSVJvvPwlSeqNoSJJ6o2hIknqjaEiSeqNoSJJ6o2hIknqjaEiSeqNoSJJ6s3/B01v63PB5joIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train.Disbursed);\n",
    "pyplot.xlabel('Disbursed');\n",
    "pyplot.ylabel('Number of customers');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "每类样本分布不是很均匀，所以交叉验证时也考虑各类样本按比例抽取"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "由于电脑老旧，故取1%的数据用于完成训练测试任务，集中精力放在参数调优的学习上。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "y = train['Disbursed']\n",
    "X = train.drop(['Disbursed'], axis=1)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.99,random_state=4)\n",
    "\n",
    "# drop ids and get labels\n",
    "y_train = y_train\n",
    "X_train = X_train.drop(['ID','DOB','LoggedIn','Unnamed: 26'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 第二轮参数调整得到的n_estimators最优值（65），其余参数继续默认值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "用交叉验证评价模型性能时，用scoring参数定义评价指标。评价指标是越高越好，因此用一些损失函数当评价指标时，需要再加负号，如neg_log_loss，neg_mean_squared_error 详见sklearn文档：http://scikit-learn.org/stable/modules/model_evaluation.html#log-loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': [0.5, 0.55, 0.6, 0.65, 0.7],\n",
       " 'colsample_bytree': [0.9, 0.95, 1.0]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "#上一轮最佳值：'colsample_bytree': 1.0, 'subsample': 0.6\n",
    "subsample = [0.5,0.55,0.6,0.65,0.7]\n",
    "colsample_bytree = [0.9,0.95,1.0]\n",
    "param_test3_1 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test3_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py:762: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.07814, std: 0.01090, params: {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  mean: -0.07655, std: 0.01033, params: {'colsample_bytree': 0.9, 'subsample': 0.55},\n",
       "  mean: -0.07651, std: 0.01215, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: -0.07732, std: 0.01507, params: {'colsample_bytree': 0.9, 'subsample': 0.65},\n",
       "  mean: -0.07481, std: 0.01338, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: -0.07709, std: 0.01521, params: {'colsample_bytree': 0.95, 'subsample': 0.5},\n",
       "  mean: -0.07511, std: 0.01512, params: {'colsample_bytree': 0.95, 'subsample': 0.55},\n",
       "  mean: -0.07842, std: 0.01219, params: {'colsample_bytree': 0.95, 'subsample': 0.6},\n",
       "  mean: -0.07569, std: 0.01488, params: {'colsample_bytree': 0.95, 'subsample': 0.65},\n",
       "  mean: -0.07591, std: 0.01505, params: {'colsample_bytree': 0.95, 'subsample': 0.7},\n",
       "  mean: -0.07650, std: 0.01332, params: {'colsample_bytree': 1.0, 'subsample': 0.5},\n",
       "  mean: -0.07539, std: 0.01393, params: {'colsample_bytree': 1.0, 'subsample': 0.55},\n",
       "  mean: -0.07181, std: 0.01353, params: {'colsample_bytree': 1.0, 'subsample': 0.6},\n",
       "  mean: -0.07438, std: 0.01487, params: {'colsample_bytree': 1.0, 'subsample': 0.65},\n",
       "  mean: -0.07466, std: 0.01777, params: {'colsample_bytree': 1.0, 'subsample': 0.7}],\n",
       " {'colsample_bytree': 1.0, 'subsample': 0.6},\n",
       " -0.07181341551570757)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb3_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=65,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=4,\n",
    "        min_child_weight=0.4,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'binary:logistic',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch3_1 = GridSearchCV(xgb3_1, param_grid = param_test3_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch3_1.grid_scores_, gsearch3_1.best_params_,     gsearch3_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.11883211, 0.1090261 , 0.11124363, 0.11395216, 0.11701207,\n",
       "        0.11216946, 0.11364298, 0.11097879, 0.112919  , 0.17567825,\n",
       "        0.13114591, 0.12134709, 0.18827295, 0.14628835, 0.15598269]),\n",
       " 'std_fit_time': array([0.00775959, 0.00307181, 0.00360588, 0.00384325, 0.00192393,\n",
       "        0.0110401 , 0.00817188, 0.00347719, 0.00333553, 0.11530976,\n",
       "        0.02382705, 0.01198668, 0.07764539, 0.02538093, 0.05434662]),\n",
       " 'mean_score_time': array([0.00643373, 0.00475793, 0.00470295, 0.00472021, 0.00476737,\n",
       "        0.00475454, 0.00499001, 0.00479031, 0.00474224, 0.00861177,\n",
       "        0.00588565, 0.00489297, 0.01175833, 0.00527034, 0.00472035]),\n",
       " 'std_score_time': array([0.00091406, 0.0001317 , 0.00014459, 0.00021009, 0.00022569,\n",
       "        0.00018378, 0.00067435, 0.0001588 , 0.00017885, 0.00474928,\n",
       "        0.00198825, 0.00033321, 0.00623147, 0.00078509, 0.00031167]),\n",
       " 'param_colsample_bytree': masked_array(data=[0.9, 0.9, 0.9, 0.9, 0.9, 0.95, 0.95, 0.95, 0.95, 0.95,\n",
       "                    1.0, 1.0, 1.0, 1.0, 1.0],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_subsample': masked_array(data=[0.5, 0.55, 0.6, 0.65, 0.7, 0.5, 0.55, 0.6, 0.65, 0.7,\n",
       "                    0.5, 0.55, 0.6, 0.65, 0.7],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.55},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.65},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.95, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.95, 'subsample': 0.55},\n",
       "  {'colsample_bytree': 0.95, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.95, 'subsample': 0.65},\n",
       "  {'colsample_bytree': 0.95, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 1.0, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 1.0, 'subsample': 0.55},\n",
       "  {'colsample_bytree': 1.0, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 1.0, 'subsample': 0.65},\n",
       "  {'colsample_bytree': 1.0, 'subsample': 0.7}],\n",
       " 'split0_test_score': array([-0.07934533, -0.07193142, -0.0663083 , -0.06641387, -0.06205785,\n",
       "        -0.06742173, -0.07166931, -0.06806308, -0.06744006, -0.06835702,\n",
       "        -0.078637  , -0.07322237, -0.06208725, -0.06184522, -0.06169938]),\n",
       " 'split1_test_score': array([-0.09035886, -0.08969928, -0.09075759, -0.09758139, -0.08948276,\n",
       "        -0.10379888, -0.09524912, -0.09511321, -0.0964832 , -0.09543884,\n",
       "        -0.09093442, -0.09585946, -0.09135121, -0.09515144, -0.0990904 ]),\n",
       " 'split2_test_score': array([-0.07740138, -0.07863317, -0.08332479, -0.0761906 , -0.07699431,\n",
       "        -0.07582137, -0.0817189 , -0.08091428, -0.08176741, -0.08168565,\n",
       "        -0.07422521, -0.06810019, -0.06707951, -0.07603997, -0.08167768]),\n",
       " 'split3_test_score': array([-0.08516124, -0.08298422, -0.08379583, -0.09007244, -0.08862781,\n",
       "        -0.07975315, -0.07766341, -0.08646143, -0.08037316, -0.08284109,\n",
       "        -0.08606957, -0.08452505, -0.08344597, -0.08465586, -0.08258829]),\n",
       " 'split4_test_score': array([-0.05829999, -0.05941977, -0.0583053 , -0.05628306, -0.05687527,\n",
       "        -0.05861197, -0.04910812, -0.06152136, -0.05228208, -0.05114457,\n",
       "        -0.05247771, -0.05513965, -0.05506277, -0.05416289, -0.04815506]),\n",
       " 'mean_test_score': array([-0.07813755, -0.07654795, -0.07650756, -0.07731991, -0.07481356,\n",
       "        -0.07709155, -0.0751077 , -0.07842219, -0.0756866 , -0.07591322,\n",
       "        -0.07649885, -0.07539013, -0.07181342, -0.07437991, -0.07465773]),\n",
       " 'std_test_score': array([0.01088089, 0.01031363, 0.01213835, 0.01506029, 0.01337424,\n",
       "        0.01520233, 0.01509901, 0.01217665, 0.01486242, 0.01503297,\n",
       "        0.01329249, 0.01391427, 0.01351714, 0.01486074, 0.01775594]),\n",
       " 'rank_test_score': array([14, 11, 10, 13,  4, 12,  5, 15,  7,  8,  9,  6,  1,  2,  3],\n",
       "       dtype=int32),\n",
       " 'split0_train_score': array([-0.02995546, -0.02408899, -0.02445397, -0.01956058, -0.0200611 ,\n",
       "        -0.02821914, -0.02497488, -0.02219536, -0.01991423, -0.01873952,\n",
       "        -0.02540238, -0.02507249, -0.02208224, -0.02015639, -0.01950099]),\n",
       " 'split1_train_score': array([-0.02609098, -0.02262993, -0.02104523, -0.01991315, -0.01813095,\n",
       "        -0.02439048, -0.02145671, -0.01987129, -0.01870622, -0.01745513,\n",
       "        -0.02666984, -0.02170056, -0.01857718, -0.01916161, -0.01729525]),\n",
       " 'split2_train_score': array([-0.02645223, -0.02507818, -0.02086644, -0.02137112, -0.01811142,\n",
       "        -0.0247523 , -0.02425693, -0.02355378, -0.02052153, -0.01921848,\n",
       "        -0.02505068, -0.02219948, -0.02080788, -0.01889624, -0.01721512]),\n",
       " 'split3_train_score': array([-0.02305203, -0.02320369, -0.02127166, -0.01921066, -0.01777215,\n",
       "        -0.02537774, -0.02137657, -0.02046404, -0.0190371 , -0.01650753,\n",
       "        -0.02409843, -0.02255493, -0.02084948, -0.01712849, -0.01836567]),\n",
       " 'split4_train_score': array([-0.0272647 , -0.02666738, -0.02393362, -0.01892497, -0.0202893 ,\n",
       "        -0.0271303 , -0.02448427, -0.02388817, -0.02000507, -0.01759676,\n",
       "        -0.02546072, -0.02189894, -0.01956533, -0.01813623, -0.01702167]),\n",
       " 'mean_train_score': array([-0.02656308, -0.02433363, -0.02231419, -0.0197961 , -0.01887298,\n",
       "        -0.02597399, -0.02330987, -0.02199453, -0.01963683, -0.01790348,\n",
       "        -0.02533641, -0.02268528, -0.02037642, -0.01869579, -0.01787974]),\n",
       " 'std_train_score': array([0.00221645, 0.00143108, 0.00154883, 0.00085457, 0.00107331,\n",
       "        0.00146515, 0.00156335, 0.00160674, 0.00066648, 0.00096664,\n",
       "        0.00082609, 0.00122799, 0.00120124, 0.00101588, 0.0009364 ])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/jhony/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.071813 using {'colsample_bytree': 1.0, 'subsample': 0.6}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3XdYVFf+x/H3GUAQxAJS7L0XsLeoMahoNLbYS+wlidnk5yYbXVPdFHfTjKYYuxFb1FgSjVii0SRGxS5iAWyoNFGkSZvz++OOZERUBIY7A+f1PDwwM3fu/Q7KfObe04SUEkVRFEXJK4PeBSiKoii2TQWJoiiKki8qSBRFUZR8UUGiKIqi5IsKEkVRFCVfVJAoiqIo+aKCRFEURckXFSSKoihKvqggURRFUfLFXu8CCkP58uVl9erV9S5DURTFphw5ciRWSunxuO2KRZBUr16doKAgvctQFEWxKUKIy7nZTl3aUhRFUfJFBYmiKIqSLypIFEVRlHxRQaIoiqLkiwoSRVEUJV9UkCiKoij5ooJEURRFyRcVJIqilxsn4PwOvatQlHxTQaIoerh9BZb3gVWD4PhqvatRlHxRQaIohS0jFX54AaQRqraHzS/D2a16V6UoeaaCRFEKW+C/4fox6PcNjFgHFZvBurFwcZ/elSlKnqggUZTCdHIdHF4E7aZCg+fAsZQWJu61YPUwuHZU7woV5YmpIFGUwhJ9Fn76B1RtB13f+/t+ZzcY+SM4u0PA8xBzTq8KFSVPVJAoSmFITYAfRkEJFxi4FOwc7n+8dAV4YZN2//f9tMZ4RbERKkgUxdKkhJ9ehZuhMHCJFho5casJozZCepIWJonRhVunouSRCpJHuJmYitEo9S5DsXWHFsLpDdBlJtTo9OhtvRrBiPWQcAMCBkDK7cKpUVHyQQXJQ2QaJaOXHmLIggNciErQuxzFVkUEab206vjDU9Ny95wqrWFIgNamsnoopCVbtkZFySeLBokQoocQ4pwQIlQIMT2Hxx2FEGtNjx8UQlQ33T9CCHHc7MsohPA1PTZMCHFKCHFSCLFdCFHeIrUDL7StzoXoRJ6du59PA89xNz3TEodSiqqkm/DDaO1SVv/5YHiCP7fafvD8Irh6UBtzkpFmuToVJZ8sFiRCCDvga6An0BAYJoRomG2z8cAtKWVt4AvgvwBSypVSSl8ppS8wCrgkpTwuhLAHvgS6SCmbAieBqZao32AQDG5Vhd3TOvOcT0W+2hOK/5x97L8QY4nDKUWN0Qg/ToSkaBi0XOuZ9aQa9YPecyB0J2yaAkb1QUaxTpY8I2kNhEopw6WUacAaoG+2bfoCy00/rwf8hBAi2zbDgHtzSAjTl4tpu9LAdUsUf497KUc+H+zLqgltMAjBqMWHeG3NMWITUy15WMXW7f8UwnZDj9lQqXne99NiNHSbpbWxbHtDa7hXFCtjySCpBFw1ux1hui/HbaSUGUA84J5tmyGYgkRKmQ68CJxCC5CGwOKCLjwn7WuX55dXO/IPvzpsPXUDv89+Y82hK6oxXnlQ2K+w5yNoOgRajsv//jq8Ck/9HwQthl//k//9KUoBs2SQZD+zAMj+rvvIbYQQbYBkKeVp020HtCBpBlREu7Q1I8eDCzFJCBEkhAiKiSmYy1FODnZM61aXX17tSD1vV6b/eEo1xiv3i78GGyaAR33o/QU8cIKdR37vQosxsP8z+GNuwexTUQqIJYMkAqhidrsyD16GytrG1P5RBogze3wof1/WAvAFkFKGSSkl8APQPqeDSykXSClbSilbenh45Od1PKC2pytrJ7XlfwObqsZ45W8ZabBujDYp45AV2uDDgiIE9PocGvWHnW/D0e8Lbt+Kkk+WDJLDQB0hRA0hRAm0UNiSbZstwGjTzwOBX00BgRDCAAxCa1u55xrQUAhxLxm6ASEWqv+RhBAMbqka4xUzu96FiEPQZx6Ur1Pw+zfYQf8FULurNsDxzOaCP4ai5IHFgsTU5jEVCER7s/9BShkshJglhOhj2mwx4C6ECAWmAeZdhDsBEVLKcLN9XgfeB/YJIU6inaF8ZKnXkBuqMV4BIHgj/PUNtJkCjQdY7jj2JWDw91C5tXYJLexXyx1LUXJJyGLQC6Rly5YyKCjI4se5m57JN3vD+HZvKM4l7JnRsz6DW1bBYCig6+SKdYq9AAueBs8GMGab9mZvaSm3YVlviAuHFzZDlVaWP6ZS7AghjkgpWz5uOzWyvQD93RjfKasxfvB3BzivGuOLrrQkbcCgXQkYtKxwQgSgZFkY9SO4esHKgRB1pnCOqyg5UEFiAbU9S2U1xofGJPLsl/v5JPCsaowvaqSEn6dBdIg2Cr1M5cI9filPGLUJHErCiv4Qd7Fwj68oJipILMS8Mb6Pb0W+3hOmGuOLmiPL4OQaeHq6NqWJHspV08IkMxVW9IOESH3qUIo1FSQWllNj/KtrjhGToBrjbdr1Y/DLv6DWM9DpDX1r8awPIzZAUqx2ZpIc9/jnKEoBUkFSSMxHxm87dQO/z/ayWo2Mt00pt7R2ERcPGLBI65art8otYOgqbc2TVYO1thtFKSQqSAqReWN8/QqlmaEa422P0Qgbp8Cd69pkjC7ZZ/TRUc3O2uqL147AmhHawEhFKQQqSHRwrzH+E9UYb3v+mAPnt0P3D62zy22D3tDnKwjfo80+rGYMVgqBChKdCCEYZGqM7+tbia/3hNH9C9UYb9Uu7tcmTWzUH9pM1ruah2s2Avw/1ka+//yamjFYsTgVJDpzL+XIZ4N9WDWhDXYG1RhvtRIiYf04cKulTYFSUJMxWkq7l6DTv7Q5uXa9q3c1ShGngsRK3GuMf9WvDr+cilSN8dYkMwPWjYW0RG0yRkdXvSvKnS7/hlYT4Y8v4fcv9K5GKcJUkFgRJwc7/q9bXba92pEGqjHeeux+H678qa1W6NlA72pyTwjo+T9oMgh2vQdBS/SuSCmiVJBYodqepVijGuOtQ8jP8OdcbYEqnyF6V/PkDAbo9y3U8ddG4Z/eoHdFShGkgsRKPawxft951RhfaOLCYdNLUMFXa7y2VXYOMHg5VGsPP06CC7v0rkgpYlSQWLmsxviJbbA3CF5YohrjC0V6ijboUAht2nYHJ70ryh+HkjBsNXg2hLUj4cpfelekFCEqSGxE+1rl2aYa4wvPtjcg8hQMWKDNZ1UUOJWBkT9CmUqwcrD2+mxEfEo6m45dY+3hKxSHpS9sjVqPxAaFRicyc+MpDl6Mo2W1cnw0oAl1vWykJ5EtOBYAm1+Gjv8Ev3f0rqbg3b4KS/whMx3GbQf3WnpXlKOYhFR2nolie3AkB8JiSc/U3qsmPFWDmb0aIKy9C3YRkNv1SFSQ2CgpJeuPRPDRthAS7mYwuXNNXnmmDk4OVjDvky2LPAWLukLlVtqsunb2eldkGTHnYWkPcHCB8YFQuqLeFQFwNS6ZwOBIdgRHcfhyHFJCNXdnejTypnsjb346cZ1lf15ibIfqvNO7oQoTC1NBYqYoBsk9cUlpfLQthPVHIqjq5swH/RrTqa7H45+oPOhuvLbSYVoyTNmvrfdRlF0/Bsue00Jk7C+6zBsmpSQ0OpHtpyMJPBPJ6Wt3AKjv7UqPxt74N/KmvrdrVmBIKfnPzyEs+eMiL7Srxvt9GqkwsSAVJGaKcpDc82dYLG9tPE14bBJ9fSvyVq+GeLg66l2W7ZASfhgFZ7fBmJ+1Hk7FwaXfIeB5rRF+9JZCGWwppeTUtXi2n45ke3Ak4THaTMXNq5bFv5EWHtXLuzzy+R//cpYF+8IZ0aYq/+nbWC1nbSEqSMwUhyABSM3I5Nu9YXyzJwwnBwMznm3AELVmfO78+RXsmAndP4D2r+hdTeE6tx3WDIfqHWD4Oov0UMs0Sg5fimP76Uh2BEdyPf4udgZB25pu9GjkTbeG3niXyf1xpZT8L/Ac3+4NY2irKnzUv4n6f24BKkjM5DlIpLT+OZVyEBajNcb/Fa4a43Pl8gFY1gvq9YQhATb5b55vJ3/QZguu31ubHr8A2oZSMzL5M/Qm209HsiskiptJaZSwN9Cpjgc9GnvjV9+Tci55X+NeSsnnO88z79dQBraozH+fb4qdCpMCZRVBIoToAXwJ2AGLpJSzsz3uCHwPtABuAkOklJeEECMA82XnmgLNgTBgv9n9lYEAKeVrj6ojz0Hy02vgVlP7hGpjby5SSjYcvcaHW8+QcDeDSZ1q8g8/1Rj/gMQY+K4j2DvB5N+0LrLF1cEF8Msb4DtCm4re8OSjA5JSM9h7LobtwZHsORtNYmoGpRzteaa+Jz0ae9O5rgcujgXXgUFKyZxdF/hy9wUGNKvEJ4N8VJgUoNwGicW6pAgh7ICvgW5ABHBYCLFFSnnGbLPxwC0pZW0hxFDgv2hhshJYadpPE2CzlPK46Tm+Zsc4AvxokReQmQ4pcbBzKUQchr5fg1NpixzKEoQQDGxRmWfqe/LRthC+2RvGzydvqMZ4c8ZM2DBOW/Fwwq7iHSIAbSbB3duw50Ptd+H/Ua4+QN1KSmNXSBSBwVHsuxBDWoYRN5cS9G5aAf9G3rSv7Y6jvWU+wAgh+L9udbEzCD7feZ5MKflskA/2dmqIXGGyZN/G1kColDIcQAixBugLmAdJX+A908/rga+EEELef5o0DFidfedCiDqAJ/efoRQcOwftFP/AV7DzXYgO0S57eNa3yOEsxc2lBJ8O8mFA80q8tfE0Lyw5pBrj79nzEVzcp31I8G6idzXWodMbWrD+9Q2UdIPOOa9HH3XnLjuCtcbyv8LjyDRKKpZxYnjrqvRo7E3LauUK9c38H351sDMIPgk8R6ZRMmeIrwqTQmTJIKkEXDW7HQG0edg2UsoMIUQ84A7Emm0zBC1wshsGrJWWvTanXdaq2AzWjYGFz0Dfr6DxAIsd0lLa1yrPL691zGqM33M2muk9GzC0VTFtjD8fCPs/hWYjtS9FI4S2+mPKbdjzAZQsC60nAnApNolAU3gcu3IbgJoeLkzuVJMejb1pUqmMrl1xX+5SG3uD4ONfzmKUki+HNsNBhUmhsGSQ5PQ/Kvub/iO3EUK0AZKllKdz2G4oMOqhBxdiEjAJoGrVqo8t9pGqPwWT92lhsn6sdqmr2yztrMWGONrb8VrXujznU5GZG0/x742n2HA0go/6N6GedzFqjL91WZu80KsJPPup3tVYH4MB+sxD3r2N2PY6v1xI5suYZpyN1JYzaFypNK93r0uPxt7U9rSu/zeTO9fCziD4YGsImcajzBvWnBL2KkwszWKN7UKIdsB7Ukp/0+0ZAFLKj822CTRtc0AIYQ9EAh73zjKEEF8AMVLKj7Lt2wdYJ6Wsm5taCqz7b0Ya7HgLDn0HVdvBoGXg6p3//eqg2DbGZ6Rq04PcDINJe612ehC9GI2SY1dvExgcya+nrjAr8T1aG87yabl38WjZl+4Nvaji5qx3mY+19I+LvP/TGbo28OTrEc0t1kZT1Onea8sUDOcBP+AacBgYLqUMNtvmZaCJlHKKqbF9gJRysOkxA3AF6HSvncXsebOBVCllrtYQLfBxJCfXwU//0AZvDVoO1doV3L4LWfaR8f/p15jORbkx/udpELQYhqyEBr31rsYqpGcaORgex/bgG+wIjiI6IRUHO0H7WuXpXd+VfidfxCE2BEZu0M7ObcT3By7xzuZgutTz4NuRLYr+hyQL0D1ITEU8C8xB6/67REr5oRBiFhAkpdwihHACVgDNgDhgqFnj/NPAbCll2xz2Gw48K6U8m5s6LDIgMSpYm4779hVtEFubKTbXRdjcgbCbzNx0ivCYJPr4VOTt3kWwMf7eWIn2r2j/ZsXY3fRM9p2PITA4il0hUcSnpFPSwY6n63ng38ibLvU9KVPSdOk26SYs7Ql3rmuj/iv6PnrnVmTlwcvM3HiaznU9+G6UCpMnZRVBYi0sNrL9bjxsfBHObYVGA6DPPHAsVfDHKSSpGZnM3xvO13tCcXIwFK3G+OgQrbNEBV9tKhAba98qCHfuprPnbDSBwZHsPRdDclompZ3s6drAC//G3nSq40HJEg95o42/Bkt6QHoSjN0OHrm6qmwV1h6+wvQfT/FU7fIsGNXy4a9ReYAKEjMWnSLFaIQ/5sCv/4HydbUuwuXrWOZYhSQsJpG3Np7mQPhNWlQrZ/uN8akJsKCLNkZi8n4oXUHvigpNbGIqu0xTsf8Rqk3F7uHqSPeGXvRo7E3bmu6579l0M0xrX7Jz1KafL1vFssUXoHVBV/nXhpO0q+nO4tGtVJjkkgoSM4Uy11b4Xlg/TmuQ7/cNNOxj2eNZmJSSH49e4wNbb4yXUvt3ObMJXtgMNTrpXZHFXbudQqBpQsSgS3EYJVRxK0mPRt70aOxNsyrl8n6WeeMkLOsNpTy0M5NSttOetvFYBP/84QSta7ixeHSrAh1hX1SpIDFTaJM2xkdoy7NeOwLt/wF+79r8ehZxSWl8vC2EdbbaGH9v2g+/d7SFqoqo0OhEAoMjCQyO5GREPAD1vFzxb6RdtmpYoXTBjfG4fABW9NfOvMf8bFMzAmw+fo3/W3ucltXcWDK2FaVUmDySChIzhTr7b0YqbJ+h9Qyq3hEGLikS61pkb4x/q3cDPF2tfB3zq4e1RuLafjB0dZ7mjrJWUkpOX7uTNUAwNDoRAN8q96Zi96KmhwXb6y7shNVDoUobrTeXQ0nLHauA/XTiOq+tPY5vlbIsG9sKV6fi116WWypIzOgyjfzxVfDz/2nTTAxeDlVaF+7xLcCmGuOTbsJ3nbTwmPQbOLvpXVG+ZRolQZfiCAyOIjA4kmu3UzAIaFPDnR6NveneyIsKZQrxDf3UetgwAer6a22DNtSB4ZdTN3hl9TEaVyrD9+NbU1qFSY5UkJjRbT2SGye1xZLir0GPj6HVBJvuInyP1TfGGzNh5SC4tB/GBUKl5npXlGdpGUb+DIvNWn72ZlIaJewMdKxTHv/G3nRt4IVbPqZiz7fDi2HrNGgyGPp/Z1NnfYHBkUxddZSGFUrz/fg2f3d3VrKoIDGj68JWKbfgx8lwIRCaDoHeX0CJh6/+Zityaox/5Zk61tEbZu9/Ye9H2u+65Ti9q3liyWkZ/Gaaiv3XkGgSUjNwKWFHF9NU7E/X87Sua/v7P4Pds6D1JOj5P5v6sLTrTBQvrTxKPW9XVoxvTVlnHUPZCqkgMaP7ColGo/bHtudDbUnTISuKzNQc2RvjR7SpipODHXYGgb1BaN/tBHYGw9+3s74bzB6//357uxy2Mwjsst1vENzfiBy6W1s6tqnpE7KNvKnFJ6ezK0TrprvvfAypGUbKOTvQraEX/o286VC7vPX2mJMSdr4Nf86Dzm9Cl3/rXdET2XM2mskBR6jtUYqVE9rka7GtokYFiZm8Bkl8ajylSxRgb5fQXdo1ZaMR+s+H+s8WzH6twIGwm7y16RRhpvW3C9O9YKlsuMl6w3RuUpax9rPJsCuJnUHgYGfIFlSPD7IHg9AUcPeFWU7hKLCze0RoZgvD8BhtRt0DYTfJMEq8Sztl9bRqXd3NdqZClxK2vALHVoD/x9DuJb0reiK/nY9h4vdB1CzvwsoJbXAvVcRmdcgjFSRm8hIkUkqGbx2ORDKiwQh6VO+BQ0E0Jt66rHURvnFc647aZSYYrPST5hMyGiUJqRlkGiUZRqP2PVOabsv77793O/P++9MzH7KdUZKZabz/dtZ3I8aMNEaceRGPlHC+rbuIqBJVnvjY9+5Pz8zh2EZJRg73F4Qa5V3wN43xaFqpjHV2XsgNY6Y2O/aZzdDvW/AdrndFT2T/hRgmLA+imrszKye0LXpTBOWBChIzeQkSozSy/vx6AkICuBh/kfIlyzOk3hAG1R2Ee0n3/BWUflcb23D0e6j5NDy/BFzyuc/i7pfpcPBbGLi00NaLkVJilNwfRJmS9CcI0fKlSlDbs5Su63gUqIxUWDVEWzBs8Pc2NzHmn6GxjFt+mMrlnFk1sY31d3G3MBUkZvLTRmKURv68/icBIQH8ce0PShhK8GzNZxnZYCT13Orlr7Ajy2HbG+Diof3RVW6Rv/0VV6d/1D4Jt5kCPf+rdzVKaiKs6Ac3TsCI9VCzs94VPZG/wm8ybtlhvMs4sXpiW7xKF98wUUFipqAa28Nvh7Pq7Cq2hG0hJSOF1t6tGdFgBJ0rd8Yur5enrh+DtS9AYqTW46XFGJtpILYKsRdgwdPg2QDGbAN71VBqFZLjYFkvbXbs0Vugkm19SDp0MY6xSw/hWVoLE+8yxTNMVJCYKeheW/Gp8fx44UdWnV1FZFIklUtVZniD4fSv3Z9SJfIwmjg5TmuED9sNviOg12c2NVJYN2lJsNAPkqK1FSzLVNa7IsVcQqQ2yePdeG1eLs/6elf0RI5cjmP0ksO4lyrB6oltqVi2+P1NqiAxY6nuvxnGDHZf2c3KkJUciz6Gi4ML/Wr3Y3j94VQt/YTL+xoz4bf/al/eTWDwCnCrUeA1FxlSwsbJ2hojIzdo06Ao1ifuojb9vBDa4NBy1fSu6IkcvXKL0YsPUdbFgdUT21K5nPWvDlmQVJCYKYxxJMGxwQSEBLD90nYyjZl0rtyZkQ1H0tq79ZM1pJ7bDhsnAQIGLIS63S1Ws00LWgo/vwZPz4Cnp+tdjfIoUWe0Oc+c3bQzE1cvvSt6Iieu3mbU4oO4OjmwZlJbm1hquKCoIDFTmAMSo5OjWXtuLevOreNW6i3qlKvDyAYjebbGszjZ5/I6a9xFWDsKok5D539pg7yKSBfhAnH9GCzurk2KOWK9TU3LUWxdPQzf9wG3mjBmK5Qsq3dFT+RURDwjFx/EpYQdqye1pZq77c9OkRsqSMzoMbI9NTOVbeHbCAgJ4Pyt85RzLMfAugMZWn8ons65mA04LRm2/hNOrILaXbWzkyIw8WC+pdzSJmM0GrV2EdVt2naE/QorB2tzn43aaHNTBQVfj2fkooM42mthUqO8bdWfFypIzOg5RYqUksORhwkICWDv1b3YCTv8a/gzssFIGpdv/Lgnw5GlsO1f2qp+g1fY1HrZBc5ohDXDtGlQxv4CVVrpXZHypII3aV21az2jTe1vY73sQm7cYcSigzjYCVZNbEstS07VbwVUkJjRfa4tk6t3rrLq7Co2hm4kKT0JXw9fRjQcQdeqXbE3PGISvoggbTR8Uiz0/hyajSy8oq3J/s9h9/taN+k2k/WuRsmro99r06k0GgDPL7K5y7bnIhMYsegvhBCsntiG2p5WNPN1AVNBYsZaguSexLRENoVuYtXZVVxNuIq3izdD6w1lYN2BlHF8yGpzSbHakrEXf4Pmo7U3U4di1Lf94j74vi807KctFqbG2ti2P+ZqEz22GKvN0mxj/54XohIYtvAgIFk1sS11vYpmmOQ2SCzaSimE6CGEOCeECBVCPNC1RgjhKIRYa3r8oBCiuun+EUKI42ZfRiGEr+mxEkKIBUKI80KIs0KI5y35GiyhVIlSjGw4kp/6/cTcLnOp5lqNOUfn0HVdV2YdmEX47fAHn+RSXruu/NQ0OLpc659/+0rhF6+HOze0EHWvDX3m2tybjpKDDv/Q/i8fWapNQW9j6ni5smZSWwxCMHTBX4TcuKN3Sbqy2BmJEMIOOA90AyKAw8AwKeUZs21eAppKKacIIYYC/aWUQ7LtpwmwWUpZ03T7fcBOSvmWEMIAuEkpYx9Vi7WdkeTkXNw5VoasZGv4VtKMaXSo2IERDUbQoVIHDCJb3p/dChunaJcEnl9ctMdQZKbD8j7aJJcTf9VGsCtFg5TaKqJHlkK3WdDhVb0remIXY5MYtuAvUjMyCZjQhkYVbWf9+tzQ/dKWEKId8J6U0t90ewaAlPJjs20CTdscEELYA5GAhzQrSgjxkfY0OdN0+ypQX0qZ6/nKbSFI7om7G8e6c+tYe24tMSkxVC9dnRENRtCnVh+cHcz6r98Mg7UjITpEm0G44z+LZjfYHW/Dn3O1XmtNB+tdjVLQjJnarA7BP8Jzc6HFaL0remKXb2phkpSWycoJbWhcqeiEiTVc2qoEXDW7HWG6L8dtpJQZQDyQvT/nEGA1gBDiXufz/wghjgoh1gkhbGt002O4Obkx2Wcygc8H8nHHj3FxcOHDgx/SdX1XPg/6nOuJ17UN3WvBhF3QZCDs+UDrzZRyW9/iC1rIz1qItByvQqSoMthpC5DV7qoNMA3epHdFT6yauwtrJ7ejlKM9wxf+xYmrRezvMBcsGSQ5XcjOfvrzyG2EEG2AZCnladNd9kBl4A8pZXPgAPBpjgcXYpIQIkgIERQTE/PExevNwc6B3jV7s7rXalb0XEH7iu35/sz39PyxJ9P2TuNo1FGkg7P2Sb3nJ9qiWQuehshTepdeMOLCYdOLULGZtt69UnTZl9C6tldubZpz7le9K3piVdycWTOpLaVLOjBy8UGOXbmld0mFyqovbQkhvgBipJQfmW4LIBFwlVIahRBVgO1SykaPqsWWLm09yo3EG6w+t5oN5zdwJ+0ODd0bMrLBSG3RrWtHYd1o7azkuTngM1TvcvMuPQUWdYP4q9qgQxubn0nJo5TbsKw3xIXBC5uhSmu9K3pi126nMGzBX8QlpbF8XCtaVCvEQcRpydoEpokxpu/RkHxTu+ydxw4q1tBGYo/W2O4HXENrbB8upQw22+ZloIlZY/sAKeVg02MG4ArQSUoZbvacNcACKeWvQogxQC8p5aBH1VJUguSe5PRkfg7/+cFFtyp1wf2naXD5d+1yUI+Pwd4GV3nbPFVbsnX4D1DXX+9qlMKUGK31SEyOg7HbwOuRnxGt0o14LUxiElJZNq41rarnMUyk1GZOTor9OxiSYv7+nvWzKTzSH9JsPCMCHPPWPVn3IDEV8SwwB7ADlkgpPxRCzAKCpJRbhBBOwAqgGRAHDL0XGkKIp4HZUsq22fZZzfScskAMMFZK+ch+sEUtSO4xSiMHrh8gICSA36/9ri26VaMnI+PjqXf4e6jUUlswq0z2pikrdiwANr+sfYrye0fvahQ93LqszRgsM7UZg21wFuyoO3cZtuAvIu/cZcmYVrStaWr6NRohJe7+QLgXBkkxZmcTpqDITM1h70IbDuDioX2V8gQXTyhlup31s6d2Ox+zB1hFkFiLohrzID5tAAAgAElEQVQk5sLjw1kV8veiW61cazDi0kmezrDDbuAS21ilLvIULOqqXdIYtcnmRjwrBSj6rDZjsKOrFialK+hd0cNlpv991mAWBslxN/jjZAgl0+Jo5paOS3qctp3MfHAfBgdTKOQQBqU8TcFh+tnZvdD+NlSQmCkOQXLPvUW3Vp9dzY2kG1QyCobfvkX/5lNx7fQv6x3Mdzde6yyQlgxT9mt/MErxdu2INoaoTBXtMldhTlqafvf+9oYHziDMLi2lxOW8D/uSZDp7cD7JievprjSqUwvvilVNweDx93cXDyhZzir/NgssSIQQtYAIKWWq6XJTU+B7KaXN9HErTkFyT4Yxg1+v/MrK4O85GnsCZ6OR/g4eDPf/mqoeVnbdWUptTMy5X7Qpxqu107sixVpc3AcBA7XF3l7YDI55nCRRSkhLfHw7Q1K0dtaQ+pCR6o6l7w8B8zDIfompRCkQgpuJqYxYdJCLsUkseKElnet65P33UcgKMkiOAy2B6kAgsAWoJ6V8tgDqLBTFMUjMBcecJmD/O2y/c55MBJ09WzCi2RTaeLd5skW3LOXPebDjLej+AbR/Re9qFGtzdqu2Pk+NjloHjHsdSKTUlhV4IAxi7g+Ge/dlpOS8/5Ju2YLhEe0NeZzfLi4pjRGLDhIWk8h3I1vQpb5tnHEXZJAclVI2F0K8AdyVUs4TQhyTUjYrqGItrbgHyT0x57axdvc/WedkIM7OQO2ytRnZYCS9avbK/aJbBe3yAVjWC+r1hCEBVnl6r1iB46u0cUUeDcDO4e+zCWPGg9sKO7OzhBzCwLztwbk82D1i5u0CdDs5jZGLD3I+MpFvRjSna0PrH0tdkEFyEK3n1UzgOSnlRSHEaSnlYxbTsB4qSMzcuUHqutFsu3WalRWqcy4jgbKOZRlUd1DuF90qKInRML8jlHCGSXvBqehMLaFYwJHlWrfwkm7ZGqKzXVoqWc5qpwuKT05n1JKDhNy4w1fDm+PfyFvvkh6pIIOkITAFOCClXC2EqAEMkVLOLphSLU8FSTaZ6bDzHeRf3xBUtRkBleux58YB7IQd3at3Z2SDkTTxaGLZGoyZ2rTwEYe1qV68LXw8RbES8SnpjF5yiNPX4pk3rBk9m1hvjzSL9NoSQpQDqkgpT+anuMKmguQhTq3XFhgqUYqrvT9hVcK5rEW3fDx8GNlgJH7V/HAwOBT8sXfPgv2fQd+vi+9CXUqxlXA3nTFLD3P86m2+HOpL76YV9S4pRwU2aaMQYq8QorQQwg04ASwVQnxeEEUqOmsyUJua3ak0VX4Yx5sZzux6fifTW08n7m4cb+x7g54berL41GLiU+ML7rjnA7UQaTZKhYhSLLk6ObB8XGuaVy3Lq2uOs/n4Nb1LypfcXNo6JqVsJoSYgHY28q4Q4qSUsmnhlJh/6ozkMe7e0Royz/4MDftC36/JdHBm/7X9BJwJ4GDkQZzsnHiu1nOMaDCCWmVr5f1Yty7Dd52gbBUYvxMcShbc61AUG5OUmsHYZYcJuhTHZ4N96N+sst4l3acgp5G3F0JUAAYDP+e7MsX6OJXWekx1mwUhP8HCZ7C7GcrTVZ5mkf8iNvTZwLM1n2Vz6Gb6be7H5J2T2RexD6M0PtlxMlK1teel1KZuUSGiFHMujvYsG9uKNjXcmfbDCdYfidC7pDzJTZDMQhs/EialPCyEqAlcsGxZSqETQluhbtQmbcK8hc9krQ1Rt1xd3m//PjsH7eSVZq9w4dYFXt79Mn039WX12dUkpyfn7hjbZ2grHfb7BtxqWvDFKIrtcC5hz5IxrehQqzxvrD/B2sO2t4S2miJFeVD8NW1K+ojD0G4qdH3/vr726Znp7Li8g4AzAZy+eRpXB1eer/s8w+oPo2KphzQanvwBfpwI7f8B3f9TSC9EUWzH3fRMJq04wr7zMXzUvwnD21TVu6QC7f5bGZgHdEBbdOp34FUppc2cg6kgyYOMNAj8NxxeCNWegoFLwPX+AVRSSk7EnCAgJIBdl3chkYxuOJppLafdv6/oEO0Mp4IvjP6p0AaAKYqtuZueyYsBR9hzLob/9G3EqHbVda2nINtIlqJNi1IRbWncn0z3KUWZfQno9am2DOq1I1oD+ZW/7ttECIGvpy+fdv6U7c9vp3fN3iwNXsr2S9v/3ig1QZveokQpGLRUhYiiPIKTgx3zR7WgawNP3t4czLI/LupdUq7kJkg8pJRLpZQZpq9lgO3MOqbkj89QbcCgQ0ltKpO/5muN5dl4u3jzXvv3aFq+KbMOzCIyKVLbbssr2op3A5eAq3WP4lUUa+Bob8c3I1rQraEX7/10hkX7wx//JJ3lJkhihRAjhRB2pq+RwE1LF6ZYEe/G2hQmtbvB9je1dbXTHlyNzcHgwOyOs8k0ZjJj/wwyD86H4I3wzNvahHuKouRKCXsD34xoTs/G3nywNYQF+8L0LumRchMk49C6/kYCN4CBwFhLFqVYoZJlYegqLRROb9AWoIoNfWCzKqWrMKPNDIKiglj212yo2wM6vKZDwYpi2xzsDMwd1oxeTSvw0bazfLP3wb83a/HYIJFSXpFS9pFSekgpPaWU/YABhVCbYm0MBuj0OozcAAmRsLALhDw4tKiv91N0T5V8VbY0wZ3/z2on0FMUa+dgZ+DLIb708anI/7afY95u6xx5kde/8GmP30Qpsmr7weTfwL0WrB0BO9+FTNN03sZMxMaJvBMTjbuTG9MPfZj7cSaKojzA3s7AF0N8GdCsEp/tPM8XO89jbcM28hokatGI4q5sVRi7HVqMgT/mQEB/bQGhfZ9A2K+U8f8vH3X+hMt3LvNJ0Cd6V6soNs3OIPhkkA8DW1Tmy90X+NzKwiSvfTGt5xUo+nFwgue+hMqt4OdpML+DtsZI06HQYgythWBs47EsOb2Epyo9hV9VP70rVhSbZWcQ/O/5ptgJwbxfQ8k0St7wr2cVq5w+9IxECJEghLiTw1cC2piSxxJC9BBCnBNChAohpufwuKMQYq3p8YNCiOqm+0cIIY6bfRmFEL6mx/aa9nnvMdtYs7IoazYSxu/QlkD1agS9P89a6XCq71QauDXgvT/fIzo5WudCFcW2GQyCjwc0YVjrqnyzN4zZv5y1ijMTi02RIoSwA84D3YAI4DAwTEp5xmybl4CmUsopQoihQH8p5ZBs+2kCbJZS1jTd3gu8LqXM9VB1NbK9kGSmgzT+vaa2ycX4iwz+aTDNPJsxv9t8DEI1vitKfhiNkne3BLPir8uMf6oGb/VqYJEzk4Ic2Z5XrYFQKWW4lDINWAP0zbZNX2C56ef1gJ948LcxDFhtwTqVgmLn8ECIANQoU4N/tf4XB24cIOBMgA6FKUrRYjAIZvVtxJj21Vn8+0Xe/+mMrmcmlgySSsBVs9sRpvty3EZKmQHEA+7ZthnCg0Gy1HRZ6+0cgkexQgPrDKRLlS7MOTqHs3Fn9S5HUWyeEIJ3n2vI+KdqsOzPS7y9+TRGoz5hYskgyekNPvurfOQ2Qog2QLKU8rTZ4yOklE2AjqavUTkeXIhJQoggIURQTEzMk1WuFDghBO+3f58yjmV4c9+bpGSk6F2Sotg8IQRv9WrA5E41CfjrCjM36RMmlgySCKCK2e3KwPWHbSOEsAfKAHFmjw8l29mIlPKa6XsCsArtEtoDpJQLpJQtpZQtPTzU1GDWoJxTOT7s8CHh8eF8HqRWa1aUgiCEYHrP+rz4dC1WH7rCjB9PFXqY5GbN9px6b10VQmw0LXL1MIeBOkKIGkKIEmihsCXbNluA0aafBwK/StOFPiGEARiE1rZyrxZ7IUR5088OQG/gNIrNaF+pPaMajmLNuTX8dvU3vctRlCJBCMG//OvxyjO1WRt0lX9tOElmIYZJbsaRfI52JrEK7VLUUMAbOAcsAZ7O6UlSygwhxFS01RXtgCVSymAhxCwgSEq5BVgMrBBChKKdiQw120UnIEJKaT71pSMQaAoRO2AXsDCXr1WxEq82f5WDNw7yzp/vsKHPBsqXLK93SYpi84QQ/LN7PewMgjm7LpBplHw6yAc7g+WbkXOzsNVBKWWbbPf9JaVsK4Q4IaX0sWiFBUB1/7U+obdCGbp1KK28W/GN3zdWMahKUYqKebsv8NnO8/Txqcjng32wt8tbK0ZBdv81CiEGCyEMpq/BZo/pPxJGsUm1y9Xmny3/ye/Xfmf1WdW7W1EK0it+dfhXj3ocvhRHbGKaxY+XmzOSmsCXQDvTXQeA/wOuAS2klL9btMICoM5IrJOUkpd3v8zBGwdZ23sttcvV1rskRSlS4lPSKVPSIc/PL7A124sCFSTWKzYllue3PE/5kuVZ1WsVjnYPDmhUFEUfBXZpSwhR2dRDK1oIESWE2CCEqFwwZSrFXfmS5flPh/9w/tZ5vjz6pd7lKIqSB7nptbUUrcfWINPtkab7ulmqKKV46VS5E8PqD2PFmRU8VfEp2ldqr3dJimL1pJQkpCcQlRRFVHLU39/Nfo5JiWHv4L3YG/I60Xvu5GbvHlLKpWa3lwkh1NqpSoGa1mIah24cYuYfM/mxz4+Ucyqnd0mKohspJbdSbz0yJKKSox6YIUIg8CjpgZeLF9VKV6O1d2vSMtOsIkhihRAj+XuE+TDgpuVKUoojJ3sn/tvpvwzbOox3/nyHuV3mqi7BSpGUacwk7m5cVihEJkfeHxBJUUQnR5NmvL+3lb2wx8PZAy9nL+q51aNT5U54OXvh5eKFl7MX3i7euJd0x8GQ98b1vMpNkIwDvgK+QOvu+ycw1pJFKcVTPbd6vNb8NT4J+oR159cxuN7gxz9JUaxIujGd2ORYopJNAWEKh+jk6L8vNyXHkCEz7nueg8EhKxSaejT9OxycvbN+dnNyw85gp9Mre7THBomU8grQx/w+06WtOZYqSim+RjYcye/XfueTw5/Q0rslNcs8ahYeRSk8aZlpD7/MZPoemxKLzDa8rqR9yayQaOXdKuvswdPZM+v+co7lbPoMPE/df4UQV6SUVS1Qj0Wo7r+2JTo5mue3PE8FlwqsfHYlDnaFf6quFC/J6cl/nznkEBBRyVHE3Y174HmuDq5ZZwxZ381/dvHC1cHVZkMit91/89oCY5u/FcUmeDp78l7793htz2vMOz6PaS2m6V2SYqOklCSmJ+Z4FmF+6SkhLeGB55Z1LJsVBo3LN34gILycvXBxcNHhVVmfvAZJ0R/FqOjKr6ofA+sOZNnpZXSo2IE2Fdo8/klKsSKlJD41PisgIpMic7z0lJyRfN/zBAL3ku54OXtR1bVq1uUm83YJD2cPnOyddHpltuehl7aEEAnkHBgCKCmltGx/sgKkLm3ZpuT0ZIb8PITkjGR+7PMjZRzL6F2SoqNj0cf44dwP94VFambqfdsYhCGr++u9y0zeLt73BYVHSQ91uTSX1BQpZlSQ2K4zN88wYtsIulTpwmedP7PZa81K/hy8cZCpu6dS0r4kNcrUyPEyk5ezF+4l3S0+ZqI4sXQbiaIUiobuDXml2St8ceQLNoVuon+d/nqXpBSyw5GHmbp7KpVdK7PYfzFuTm56l6RkY8mldhWlQIxpNIbW3q35+NDHXLlzRe9ylEIUFBnEy7tfplKpSizqvkiFiJVSQaJYPYMw8OFTH+JgcGD6/umkG9P1LkkpBEeijvDS7peo4FKBRf6LcC/prndJykOoIFFsgreLN++0e4dTsaeYf2K+3uUoFnYs+hgv7noRbxdvFvsvVssxWzkVJIrN8K/uT7/a/Vh0ahFHoo7oXY5iIcejjzNl5xS8nL1Y3F2FiC1QQaLYlOmtp1OpVCVm7J/BnbQ7epejFLATMSeYsmsKHs4eLPZfjIezh94lKbmggkSxKS4OLszuOJvo5Gg+OPABxaH7enFxMuYkU3ZOwd3JncXdF+Pp7Kl3SUouWTRIhBA9hBDnhBChQojpOTzuKIRYa3r8oBCiuun+EUKI42ZfRiGEb7bnbhFCnLZk/Yp1aurRlBd9XuSXS7/wc/jPepejFIDTsaeZvHMy5ZzKsdh/MV4uXnqXpDwBiwWJEMIO+BroCTQEhgkhGmbbbDxwS0pZG22a+v8CSClXSil9pZS+wCjgkpTyuNm+BwCJlqpdsX4TmkyguWdzPjz4IREJEXqXo+RDcGwwk3ZMoqxjWZb4L8HbxVvvkpQnZMkzktZAqJQyXEqZBqwB+mbbpi+w3PTzesBPPDh0eRh/L6qFEKIUMA34wCJVKzbBzmDHRx0/QiCYsX8GGcaMxz9JsTpnbp5h4s6JlHYsrULEhlkySCoBV81uR5juy3EbKWUGEA9k7yw+BLMgAf4DfAYkoxRrlUpV4q22b3E85jgLTy3UuxzlCYXcDGHijom4OriyxH8JFUpV0LskJY8sGSQ5TYqUvWX0kdsIIdoAyVLK06bbvkBtKeXGxx5ciElCiCAhRFBMTMwTlK3Ykl41e9G7Zm++O/Edx6OPP/4JilU4F3eOiTsn4uLgwmL/xVQsVVHvkpR8sGSQRABVzG5XBq4/bBshhD1QBjBfPWYo95+NtANaCCEuAb8DdYUQe3M6uJRygZSypZSypYeH6kJYlP27zb/xdvFm+v7pJKappjNrdy7uHBN2TKCkfUkW+y+msmtlvUtS8smSQXIYqCOEqCGEKIEWCluybbMFGG36eSDwqzT15xRCGIBBaG0rAEgpv5VSVpRSVgeeAs5LKZ+24GtQbIBrCVc+7vgxN5Ju8PGhj/UuR3mE87fOM3HHRBztHFnSfQlVXKs8/kmK1bNYkJjaPKYCgUAI8IOUMlgIMUsIcW8N+MWAuxAiFK0B3byLcCcgQkoZbqkalaKjmWczJjWdxJawLWy/uF3vcpQcXLh1gQmBE3Cwc2CJ/xKqlFYhUlSo9UiUIiPDmMHo7aO5GH+RDc9tUI23ViTsdhjjAsdhJ+xY2mMp1UpX07skJRdyux6JGtmuFBn2BntmPzWbTGMmM36fQaYxU++SFCD8djjjA8djJ+xY7L9YhUgRpIJEKVKqlK7Cv9v8myNRR1gavFTvcoq98PhwxgWOQwjBIv9F1ChTQ++SFAtQQaIUOX1q9cG/uj9fH/ua07FqFh29XIy/yPjA8QAs7r6YmmVq6lyRYikqSJQiRwjB223fxr2kO9P3Tyc5XY1dLWyX71xmfOB4jNLIYv/F1CyrQqQoU0GiFEllHMvwcUdtad7/Hf6f3uUUK1fuXGFc4DgyZSaLui+iVtlaepekWJgKEqXIauXdinGNx7HhwgZ2Xd6ldznFwtU7VxkXOI70zHQWdl9InXJ19C5JKQQqSJQi7WXfl2no3pD3DrxHVFKU3uUUaVcTrjJuxzhSM1NZ2H0hdcvV1bskpZCoIFGKNAc7B2Z3nE1aZhoz/5iJURr1LqlIupZ4jfGB40nJSGFh94XUc6und0lKIVJBohR5NcrU4M1Wb3LwxkFWnFmhdzlFzvXE64zbPo6k9CQWdltIfbf6epekFDIVJEqxMKDOAPyq+jHn6BzOxp3Vu5wi40biDcYFjiMhPYGF3RfSwL2B3iUpOlBBohQLQgjea/cebo5uvLnvTVIyUvQuyeZFJkUyNnAsd1LvsLDbQhq6Z18AVSkuVJAoxUZZp7J88NQHhMeH81nQZ3qXY9MikyIZu30s8anxLOi+gEblG+ldkqIjFSRKsdKuYjtGNxzN2nNr2Xt1r97l2KSopCjGBY7jduptvuv2HY3LN9a7JEVnKkiUYucfzf9BvXL1eOePd4hNidW7HJsSnRzN+B3jibsbx/xu82nq0VTvkhQrYK93AXpJT08nIiKCu3fv6l2KooN3ar1DbHIsZ0PO4lbSDZHjqs/Ww8nJicqVK+Pg4KBbDTHJMYwPHE9McgzfdfsOHw8f3WpRrEuxDZKIiAhcXV2pXr06Qlj3m4hiGXEpcdxIuoGXixfuJd31LuehpJTcvHmTiIgIatTQZ/bc2JRYxgWOIyo5iu+6fYevp68udSjWqdhe2rp79y7u7u4qRIqxck7lKFWiFFHJUdzNsN4zUyEE7u7uup09m4fIt12/pZlnM13qUKxXsQ0SQIVIMSeEoFKpShiEgYjECKse9a7X/9WbKTeZEDiByKRIvvH7hhZeLXSpQ7FuxTpIFMXeYE+lUpVIzUglKlnNxWUu7m4cE3ZM4FriNb72+5qW3o9dcVUpplSQ6OT27dt88803eXrunDlzSE4uuDU2li1bxtSpUwtsf7m1d+9eevfu/cTPe9J6L126xKpVqx76uGsJV9yc3IhLiSMhLeGJ64mLi6Nbt27UqVOHbt26cevWrRy3e/PNN2ncuDGNGzdm7dq1T3ycwnQvRCISIvjK7ytaebfSuyTFiqkg0Yk1BUlR96ggycjIAMDLxQtHe0euJ14nw5jxRPufPXs2fn5+XLhwAT8/P2bPnv3ANlu3buXo0aMcP36cgwcP8sknn3Dnzp0nfzGF4NbdW0zcMZErd64wz28ebSq00bskxcpZNEiEED2EEOeEEKFCiOk5PO4ohFhrevygEKK66f4RQojjZl9GIYSv6bHtQogTQohgIcR8IYSdJV+DpUyfPp2wsDB8fX154403+OSTT2jVqhVNmzbl3XffBSApKYlevXrh4+OT9Sl27ty5XL9+nS5dutClS5eH7n/79u00b94cHx8f/Pz8AO2Tc79+/WjatClt27bl5MmTDzxv3bp1NG7cGB8fHzp16gRob8QdO3akefPmNG/enD///BPQzig6d+7M4MGDqVu3LtOnT2flypW0bt2aJk2aEBYWBsCYMWOYMmUKHTt2pG7duvz8888PHDcpKYlx48bRqlUrmjVrxubNmx/5+7t69So9evSgXr16vP/++wC8/fbbfPnll1nbzJw5k7lz5zJ9+nT279+Pr68vX3zxBcuWLWPQoEE899xzdO/eHYDPPv2MIV2H0KdjH17/9+tIKQEICAigdevW+Pr6MnnyZDIzMx+oZfPmzYwePRqA0aNHs2nTpge2OXPmDJ07d8be3h4XFxd8fHzYvn37I1+jHm7fvc2knZO4FH+Juc/MpW2FtnqXpNgAi3X/Nb3Bfw10AyKAw0KILVLKM2abjQduSSlrCyGGAv8FhkgpVwIrTftpAmyWUh43PWewlPKO0Fof1wODgDX5qfX9n4I5c71gPx02rFiad597+LQRs2fP5vTp0xw/fpwdO3awfv16Dh06hJSSPn36sG/fPmJiYqhYsSJbt24FID4+njJlyvD555+zZ88eypcvn+O+Y2JimDhxIvv27aNGjRrExcUB8O6779KsWTM2bdrEr7/+ygsvvMDx48fve+6sWbMIDAykUqVK3L59GwBPT0927tyJk5MTFy5cYNiwYQQFBQFw4sQJQkJCcHNzo2bNmkyYMIFDhw7x5ZdfMm/ePObMmQNoYfTbb78RFhZGly5dCA0Nve+4H374Ic888wxLlizh9u3btG7dmq5du+Li4pLjazx06BCnT5/G2dmZVq1a0atXL8aPH8+AAQN49dVXMRqNrFmzhkOHDtG0aVM+/fTTrABbtmwZBw4c4OTJk7i5ubFjxw4uXLhA0OEgYpNjGdh/IFt3b6VWpVqsXbuWP/74AwcHB1566SVWrlzJCy+8wIQJE5gyZQotW7YkKiqKChUqAFChQgWio6MfqNfHx4f333+fadOmkZyczJ49e2jY0LrmpopPjWfSzkmE3w5n7jNzaV+xvd4lKTbCkuNIWgOhUspwACHEGqAvYB4kfYH3TD+vB74SQgh57+OgZhiw+t4NKeW9d3x7oARgvq1N2rFjBzt27KBZM61bZWJiIhcuXKBjx468/vrrvPnmm/Tu3ZuOHTvman9//fUXnTp1yhpz4ObmBsDvv//Ohg0bAHjmmWe4efMm8fHx9z23Q4cOjBkzhsGDBzNgwABAG7w5depUjh8/jp2dHefPn8/avlWrVllvorVq1cr6hN+kSRP27NmTtd3gwYMxGAzUqVOHmjVrcvbs/TPw7tixgy1btvDpp58CWvfsK1eu0KBBzrPJduvWDXd3bezHgAED+P3333nttddwd3fn2LFjREVF0axZs6xtcnr+vd9L9t//7Tu3ORZ8jNDgUI4cOUKrVlr7QEpKCp6engAsWrQox/0+TPfu3Tl8+DDt27fHw8ODdu3aYW9vPcO44lPjmbhjIqG3Q5n7zFw6VOqgd0mKDbHk/+RKwFWz2xFA9outWdtIKTOEEPGAO2A+b8UQtMDJIoQIRAuqX9AC6AFCiEnAJICqVas+stBHnTkUBiklM2bMYPLkyQ88duTIEbZt28aMGTPo3r0777zzTq72l1N30fvzWZN9u/nz53Pw4EG2bt2Kr68vx48fZ968eXh5eXHixAmMRiNOTk5Z2zs6Omb9bDAYsm4bDIas9oecjpP9tpSSDRs2UK9e7hZEetj+JkyYwLJly4iMjGTcuHEPfb75mU723396Zjph8WGsXrSaF154Icc2D3NeXl7cuHGDChUqcOPGjaywyW7mzJnMnDkTgOHDh1OnjnUsQ3sn7Q6Tdk4i9HYoc7rM4alKT+ldkmJjLNlGklPH9+zvZI/cRgjRBkiWUp6+bwMp/YEKgCPwTE4Hl1IukFK2lFK29PDweKLCC4OrqysJCVoPIX9/f5YsWUJiYiIA165dIzo6muvXr+Ps7MzIkSN5/fXXOXr06APPzUm7du347bffuHjxIkDWpa1OnTqxcuVKQGvfKF++PKVLl77vuWFhYbRp04ZZs2ZRvnx5rl69Snx8PBUqVMBgMLBixYoc2wkeZ926dRiNRsLCwggPD38gMPz9/Zk3b15W2B07duyR+9u5cydxcXGkpKSwadMmOnTQPkH379+f7du3c/jwYfz9/YHH/76y//6jI6NxSHKgRYcW/LD+h6xLVXFxcVy+fPmB5/fp04fly5cDsHz5cvr27fvANpmZmdy8eROAkydPcvLkyayzNz0lpCUwecdkzt86zxdPf0Gnyp30LniEQDkAABdUSURBVEmxQZY8I4kAqpjdrgxcf8g2EUIIe6AMEGf2+FDMLmuZk1LeFUJsQTtb2VlQRRcWd3d3OnToQOPGjenZsyfDhw+nXbt2AJQqVYqAgABCQ0N54403MBgMODg48O233wIwadIkevbsSYUKFe67fHSPh4cHCxYsYMCAARiNxqw2jvfee4+xY8fStGlTnJ2ds978zL3xxhtcuHABKSV+fn74+Pjw0ksv8fzzz7Nu3Tq6dOny0HaLR6lXrx6dO3cmKiqK+fPn33dWA1pD+WuvvUbTpk2RUlK9evUcG+Xveeqppxg1ahShoaEMHz6cli21MQ4lSpSgS5culC1bFjs7rR9G06ZNsbe3x8fHhzFjxlCuXLn79tW9e3dCQkIe+P239GnJS2++RNduXUGCg4MDX3/9NdWqVbuvjWT69OkMHjyYxYsXU7VqVdatWwdAUFAQ8+fPZ9GiRaSnp2ddmixdujQBAQG6X9pKSEtgys4pnL11li+e/oLOVTrrWo9iw6SUFvlCC6lwoAZaW8YJoFG2bV4G5pt+Hgr8YPaYAS1oaprdVwqoYLb/tcDUx9XSokULmd2ZM2ceuE+xjNGjR8t169YVyrEyMzOlj4+PPH/+fP73ZcyU5+POy7M3z8r0zPQCqC5/CvL/bEJqghy+dbj0Xe4rd1/eXWD7VYoWIEjm4v3eYpe2pJQZwFQgEAgxhUSwEGKWEKKPabPFgLsQIhSYBph3Ee4EREhTY72JC7BFCHESLZiigfmWeg2KbTlz5gy1a9fGz8+vQNofDMJAZdfKZMpMridez7GNyRYlpSfx4q4XORN7hk87f8ozVXO8OqwouSaKyh/Ho7Rs2VLe6656T0hIyEN7BNmSNm3akJqaet99K1asoEmTJjpVVHACAwN5880377uvRo0abNy4sVDriE2JJSopioqlKlLOqdz/t3fv0VHV1wLHv5uQJ2AkJDxCQBBfRaCpClatVGspWKwiIoJUQEGXpQJloZVHqUC5AgsXFaG9Xnpr8VK0GqrIFXlYEhqvRQfUJDzL20USAuERNAl5/+4f50yYhISZyczJJGR/1srKzDnn95udk9/MnnPOzG97b+CQYIxZdxLJys9iyQ+XMOiaQUGKTl2JROQLY4zXuXGazucPVYN8/vnnoQ7BMYMHD66+YB5KHaI6UFhWSF5RHjGtY4hsHem9URNUXF7MpH9MIis/i8UDF2sSUUGjU6Qo5YV7lmBBmvwswfUpLi9m0tZJZOZnsmjgIgb3CH2CVlcOTSRK+SA8LJzEtomUVJSQX5wf6nD8cqHiAs+lPsdXp75i4d0LGdJjSKhDUlcYTSRK+eiqyKu4OupqTl84TVF5UajD8cmFigtM3jqZL05+wcs/eJn7e94f6pDUFUgTiVJ+6BzTmYiwCHK+zaGyyv8vZjamkooSJqdOxpXnYsFdCxh67dBQh6SuUJpIQqQpTSPf0uuR+COsVRhJbZOoqKogt8j6SHCg9UjGjx9Pz549SU5Orp6WJlAlFSVMSZ2C64SLBT9YwM96/SzgPpWqjyaSEGlKieRK50s9En9Eh0eTEJPAN6XfcL70fFDqkSxZsoSMjAwyMjJITk72OyZPpZWlTE2bymcnPmP+XfN5sNeD3hspFQD9+C/AxhmQtyu4fXbuC/fXP9mfZz2SQYMG0bFjR959911KS0t5+OGHmTdvHkVFRYwcOZLs7GwqKyuZM2cOJ0+erK5HEh8fX+cUKWDVI5k1axaVlZXEx8ezdetWzp49y1NPPcWRI0eIiYlh5cqV9OvXr0a7lJQU5s2bR1hYGLGxsaSnp3Ps2DGeeOIJioqs6wIrVqzgzjvvZNu2bbz00kt06tSJjIwMhg8fTt++fVm2bFn1HFi9evVi/PjxREVFsWfPHk6ePMnSpUsvORIpKipi8uTJ7Nq1i4qKCubOnVvnnFVu7nokR48e5fHHH+ell15izpw5xMfHM3XqVMCaJLFTp0689dZb7Nu3j+TkZMaNG0f79u3ZsGEDJSUlFBUVkZqaypIlSy7Z/2DVI3nttdcoKyvj9ttv549//CNhYWHER8dTWF7IiaITrPtgHf/c9k/Aqkdyzz33sHjx4hrxetYjcU/XsmnTJkaOHFnv39gQ7iTyr9x/Mf/O+Qy7blhQ+1eqLnpEEiKLFi2iV69eZGRkMGjQIA4ePIjL5SIjI4MvvviC9PR0Nm3aRGJiIpmZmezevZshQ4YwZcoUEhMTSUtLqzeJuOuR/P3vfyczM7N67id3PZKsrCxefvllxo4de0lbdz2SzMxM1q9fD1ysR/Lll1/yzjvvMGXKlOrtMzMzWbZsGbt27WL16tUcOHAAl8vFxIkTWb58efV27nokGzZs4Nlnn6WkpKTG47rrkezYsYO0tDReeOGF6sRVF5fLxZo1a8jIyCAlJYWdO3cyYcKE6vnD3PVIxowZw6JFi7j77rvJyMhg2rRpAGzfvp0333yT1NTU6noktff/vn37quuRuKfQd096+fTTT5O3Pw+AvLw8OnfuDFy+HsnGjRspLi7m9OnTpKWlcfz4xcmxZ8+eTb9+/Zg2bdolXzD1VVllGdPSpvFpzqfMvWMuD1//cIP6UcpfekQClz1yaAxaj6Rp1SNx7/+srCyv9UjOl1r7L/9CPh1j6p4+Hi5fj2ThwoV07tyZsrIynnnmGRYvXuxTuQBPZZVlTNs2jU9yPuG3d/yWR254xK/2SgVCE0kTYLQeSZOqR+K2fPlyxo0bx8KFC+vtJzYyloSOCew9spe2N7bl/OnzftcjcSfiyMhInnzyyepk6qvyynKmb5tOenY6c74/h0dveNSv9koFSk9thYjWI2m69Ujc+/++++5j7dq1XuuRDHtoGB+mfEh2YTZ/WfUXv+uRnDhxArAS2rp16+jTp89l/3ZP5ZXlTP/ndLZlb2P27bMZeWNwr7ko5Qs9IgkRrUfS9OuR9O7dmwULFvCTn/yEqqqqeuuRzJo5i0cefYSU1Sl0696N9e9Z15Z8rUcyZswY8vPzMcaQnJzM66/7NqF1eVU5L6S/QNrxNGYOmMmom0b51E6pYNPZf5Xjxo8fzwMPPMCIESMcf6yqqipuueUWUlJSGr2U7aniU+QX55PULonYyNig9+85Zsurynkx/UU+/vpjZgyYwZjvjAn64ynl6+y/empLXTGCXY/EXwnRCUSHR5NbmEtZZZljj1NRVcGM9Bl8/PXH/Lr/rzWJqJDTU1vNXHOoR7Jq1aoGtfO3Hknv3r05cuRInesag4iQ1DaJwwWHySnMocdVPer80EMgKqoqmPnJTLZ8vYXnb3ueJ3o/EdT+lWoITSTNnNYjaVoiwiLo0qYLOYU5nL5wmoSYhKD1bYxh1iez2HRsE9Nvnc64m8cFrW+lAqGntpQKstjIWGIjY8kvzqe4PDhT2RhjKCgtYOOxjUy7dRrj+4wPSr9KBYMmEqWCTETo0qYLrVu1Jqcw8FmCjTHkFOZwoeICU2+ZylN96v9+jFKhoIlEKQeEtQqja7uulFWWkVec1+B+jDHkFuZyvvQ87SLaMbHvxCBGqVRwaCJRyiFtwtsQHx1PQUlB9VQq/nAnkYLSAhJiEmgX0c6BKJUKnKOJRESGiMi/ReSQiMyoY32kiLxjr/9cRHrYy8eISIbHT5WIJItIjIhsEJH9IrJHREI7SVYAmtI08lqPJDApKSncfPPNtGrVitrfV0qISSCqdRQnCk/w4YYPufHGG7nuuuvqnGrekzGG3KKLSeRy83gpFWqOJRIRCQP+ANwP9AZGi0jvWptNAM4ZY64Dfg8sBjDGrDHGJBtjkoEngGPGGHe1n1eMMTcB3wPuEpFmWTu0KSWSK12w65HU1qdPH9577z0GDhx4ybpW0oqkdklUVFbwy+d+yUcffcTevXt5++232bt3b539GWM4UXSCghJNIqp5cPLjvwOAQ8aYIwAi8jfgIcDz2fMQMNe+vRZYISJian7dfjTwNoAxphhIs2+XiciXQFKggS52LWb/2f3eN/TDTXE38eKAF+tdr/VImnc9Ek/eZkiIDIskZ28OST2SiE2MJSIiglGjRvHBBx/Qu3fN91buJHKu5Bzx0fEkRAfv48NKOcXJRNIVOO5xPxu4vb5tjDEVInIe6ACc9tjmMayEU4OIXA38DFhW14OLyDPAMwDdu3dv2F/goEWLFrF7924yMjLYsmULa9euxeVyYYzhwQcfJD09nfz8fBITE9mwYQMA58+fJzY2lqVLl5KWlkZ8fHydfbvrkaSnp9OzZ8/qSRvd9UjWrVtHamoqY8eOvaSsq7seSdeuXSkoKAAu1iOJiori4MGDjB49uvoUTmZmJvv27SMuLo5rr72WiRMn4nK5WLZsGcuXL+fVV18FLtYjOXz4MPfeey+HDh2q8bjueiRvvPEGBQUFDBgwgB//+Mf1zuvlcrnYvXs3MTEx9O/fn6FDhzJhwgSGDx/O1KlTq+uRuFwu+vXrxyuvvFI9d9eqVavYvn07WVlZxMXF1ahH4rn/ExISquuRhIeHM2nSJNasWcPYsWNrzLXli2/yv6F7t+6cKj5Fm/A2JCUlXfIdoNpJpGNMx6B/oVEpJziZSOp6BtSe2Ouy24jI7UCxMWZ3jUYirbGOUl5zH/Fc0okxK4GVYM21dblAL3fk0Bi0HknzrUfij+jwaMIkjOxvs6kyVTWShCYR1Zw5mUiygW4e95OA3Hq2ybaTQyxw1mP9KOzTWrWsBA4aY14NXriho/VImm89El8lJSWRk51D17Zd+fqbr9l/ZD+JiYnVj59XlMe5knN0iO6gSUQ1O05+amsHcL2I9BSRCKyksL7WNusB9zwPI4BU9/UREWkFPAr8zbOBiCzASji/cjB2x2k9kiunHokv+vfvz8GDB8nPyaddWDveW/sePxryIyuJFOdxtuQsHaI70CmmkyYR1ew4dkRiX/N4DtgMhAFvGGP2iMh8YKcxZj3wZ2C1iBzCOhLxLKgwEMj2PHUlIknAbGA/8KX9hFthjPH/PEOIaT2SK6ceyfvvv8/kyZPJz89n6NChJCcns3nzZnJzc5k4cSIfffQRrVu3ZsWKFQwePJjKykqGjR5G7DWx1aez4qLjNImoZkvrkSjHtZR6JP4orSjl8PnDGGOIi46jc0xnr0lEx6xqbFqPRLU4oa5H4o/I1pF0a9eNTm06+ZRElGrKdBr5Zk7rkVwU6nok/moX0Y526LQnqvnTRNLMaT0SpVSotehTWy3h+pC6MuhYVU1Zi00kUVFRnDlzRp+gqskzxnDmzJlLPummVFPRYk9tJSUlkZ2dTX5+fqhDUcqrqKgokpICnlZOKUe02EQSHh5ePYWIUkqphmuxp7aUUkoFhyYSpZRSAdFEopRSKiAtYooUEckHGjbbHsRTsz5KU6Fx+Ufj8o/G5Z8rNa5rjDFeq6u1iEQSCBHZ6ctcM41N4/KPxuUfjcs/LT0uPbWllFIqIJpIlFJKBUQTiXcrQx1APTQu/2hc/tG4/NOi49JrJEoppQKiRyRKKaUC0qISiYgMEZF/i8ghEZlRx/rxIpIvIhn2z0SPdeNE5KD9M85j+a0issvu8zVpQIWihsYlIskisl1E9ohIlog85tFmlYgc9WiT3Fhx2esqPZav91jeU0Q+t/fjOyIS0Vhxici9HssyRKRERIbZ6wLeX77EZm8zUkT22v+3tzyWOzLGGhqT0+MrkNjs5SEbY/XF5fQY82Hs/96j/wMiUuCxzrHXMIwxLeIHq278YeBaIALIBHrX2mY8Vg342m3jgCP27/b27fb2OhdwByDARuD+RozrBuB6+3YicAK42r6/ChgRiv1lryusZ/m7wCj79uvALxozrlr/07NATDD2lx+xXQ985TF+Ojo5xgKMybHxFWhsTWCM1RuXU2PMl7hqbT8ZeMPJ8eX+aUlHJAOAQ8aYI8aYMuBvwEM+th0MfGyMOWuMOQd8DAwRkS7AVcaY7cb6j/wPMKyx4jLGHDDGHLRv5wKnAK9fHnI6rvrY73R+BKy1F71JI+6vWkYAG40xxQ1oG0hsTwN/sMcRxphT9nKnxliDY3J4fAUUW30acYz5Elewx5i/Y3808LZ928nXsBaVSLoCxz3uZ9vLanvEPoxfKyLdvLTtat/21qdTcVUTkQFY71IOeyz+D7vN70UkspHjihKRnSLymfvQHugAFBhjKrz06WRcbqO4+CRzC2R/+RrbDcANIvKpvW+GeGkb6BgLJKZqDoyvYMQWyjHmdZ8R/DHm69hHRK4BegKpXtoG4zWsRSWSus771f7I2v8CPYwx/YB/YL2buVxbX/p0Mi6rA+tdxWrgSWNMlb14JnAT0B/rcLZm8XPn4+purG/UPg68KiK9fOzT6bjc+6svsNljcaD7y9fYWmOdFrkH6x3jf4vI1ZdpG+g+CyQmqwNnxlcwYgvlGPNlnwV7jPnzt40C1hpjKr20Dcb+alGJJBvwfGeaBOR6bmCMOWOMKbXv/gm41UvbbPt2vX06HBcichWwAfiNMeYzjzYnjKUU+AvWYXGjxWWfCsEYcwTYBnwPa86fq0XEXQen0feXbSTwvjGm3KNNoPvLp9jsbT4wxpQbY44C/8Z6QXJqjAUSk5PjK+DYQjnGLheXzYkx5ktcbrWPhpx8DWtRF9tbY11g6snFC1U319qmi8fth4HPzMULVUexLlK1t2/H2et2AN/n4oWqnzZiXBHAVuBXdfTbxf4twKvAokaMqz0Qad+OBw5iXxQEUqh5IXRSY8Xlsewz4N5g7i8/YhsCvOmxb45jnY5xZIwFGJNj4ysIsYV6jNUZl5NjzJe47O1uBI5hf0/QXubYa5gxpuUkEnuH/RQ4gHWed7a9bD7woH17IbDH/gelATd5tH0KOGT/POmx/DZgt93nCs9/ntNxAT8HyoEMj59ke10qsMuO7a9A20aM6077sTPt3xM8+rwW61Mih7Ce8JGN/H/sAeQArWr1GfD+8jE2AZYCe+3HG+X0GGtoTE6PrwBjC/UYu9z/0bEx5i0u+/5c6khSTo0vY4x+s10ppVRgWtI1EqWUUg7QRKKUUiogmkiUUkoFRBOJUkqpgGgiUUopFRBNJEoFSETmisjzTSCOYyISH+o4VMujiUQppVRANJEoVQcRaSMiG0QkU0R2i8hjnu/4ReQ2Ednm0eS7IpJq13p42t6mi4ik27UhdovI3fby/7QnG9wjIvM8HvOYiLwsVg2QnSJyi4hsFpHDIvKsvc09dp/vi1UL43URueR5LCI/FxGX/dj/JSJhTu4v1bJpIlGqbkOAXGPMd40xfYBNXrbvBwzFquvwWxFJxJpMcLMxJhn4LtY3w8H6RvJtdpsfikg/j36OG2PuAD7Brl+BNX3FfI9tBgDTsSYF7AUM9wxERL4DPAbcZT92JTDGj79dKb+09r6JUi3SLuAVEVkMfGiM+cRL4bgPjDEXgAsikob1Yr8DeENEwoF1xhh3IhkpIs9gPf+6AL2BLHudu9LfLqwpNL4FvhWr0p57dlmXsSYqRETeBn7AxfobAPdhTVS5w445GquWiFKO0ESiVB2MMQdE5FasuY0WisgWoIKLR/FRtZtc2oVJF5GBWEcqq0VkCdaRxvNAf2PMORFZVasv96zFVR633ffdz9dLHqvWfcGaUHCmlz9TqaDQU1tK1cE+NVVsjPkr8ApwC9aMqu4p6R+p1eQhEYkSkQ5YNSp22MWFThlj/gT82e7jKqAIOC8inYD7GxDeALHqkrfCOoX1f7XWbwVGiEhH+2+Js2NRyhF6RKJU3foCS0SkCmsG3F9gnSL6s4jMAj6vtb0Lq25Hd+B3xphcERkHvCAi5UAhMNYYc1REvsKanfgI8GkDYtsOLLJjTAfe91xpjNkrIr8BttjJphz4JfB1Ax5LKa909l+lmhERuQd43hjzQKhjUcpNT20ppZQKiB6RKKWUCogekSillAqIJhKllFIB0USilFIqIJpIlFJKBUQTiVJKqYBoIlFKKRWQ/wc2HtgizHR30wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch3_1.best_score_, gsearch3_1.best_params_))\n",
    "test_means = gsearch3_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch3_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch3_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch3_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch3_1.cv_results_).to_csv('my_preds_subsampleh_colsample_bytree_2.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(colsample_bytree), len(subsample))\n",
    "train_scores = np.array(train_means).reshape(len(colsample_bytree), len(subsample))\n",
    "\n",
    "for i, value in enumerate(colsample_bytree):\n",
    "    pyplot.plot(subsample, -test_scores[i], label= 'test_colsample_bytree:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'subsample' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'subsample_vs_colsample_bytree2.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "最佳值：'colsample_bytree': 1.0, 'subsample': 0.6"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
